{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W0711 17:01:19.512768 11372 deprecation.py:323] From <ipython-input-3-c113f232fe84>:1: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "W0711 17:01:19.522741 11372 deprecation.py:323] From D:\\Anaconda\\1\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "W0711 17:01:19.525733 11372 deprecation.py:323] From D:\\Anaconda\\1\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting C:\\Users\\ASUS\\Desktop\\AI学习\\tensorflow\\train-images-idx3-ubyte.gz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0711 17:01:19.854885 11372 deprecation.py:323] From D:\\Anaconda\\1\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "W0711 17:01:19.858843 11372 deprecation.py:323] From D:\\Anaconda\\1\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.one_hot on tensors.\n",
      "W0711 17:01:19.926663 11372 deprecation.py:323] From D:\\Anaconda\\1\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting C:\\Users\\ASUS\\Desktop\\AI学习\\tensorflow\\train-labels-idx1-ubyte.gz\n",
      "Extracting C:\\Users\\ASUS\\Desktop\\AI学习\\tensorflow\\t10k-images-idx3-ubyte.gz\n",
      "Extracting C:\\Users\\ASUS\\Desktop\\AI学习\\tensorflow\\t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "mnist = input_data.read_data_sets(r'C:\\Users\\ASUS\\Desktop\\AI学习\\tensorflow',one_hot=True,source_url='http://yann.lecun.com/exdb/mnist/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0711 17:08:40.488051 11372 deprecation.py:323] From <ipython-input-16-c9303f17fa03>:19: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See `tf.nn.softmax_cross_entropy_with_logits_v2`.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "learning_rate=tf.placeholder(tf.float32)\n",
    "def initialize(shape,stddev=0.1):\n",
    "    return tf.truncated_normal(shape,stddev=0.1)#设定初始化\n",
    "#l1层\n",
    "L1_units_count=100\n",
    "x=tf.placeholder(tf.float32,[None,784],name='x')\n",
    "w_1=tf.Variable(initialize([784,L1_units_count],stddev=0.05))\n",
    "b_1=tf.Variable(initialize([L1_units_count]))\n",
    "logits_1=tf.matmul(x,w_1)+b_1\n",
    "output_1=tf.nn.relu(logits_1)#使用relu激活函数\n",
    "#l2层\n",
    "L2_units_count=10\n",
    "w_2=tf.Variable(initialize([L1_units_count,L2_units_count],stddev=0.063))\n",
    "b_2=tf.Variable(initialize([L2_units_count]))\n",
    "logits_2=tf.matmul(output_1,w_2)+b_2\n",
    "\n",
    "logits=logits_2\n",
    "y=tf.placeholder(tf.float32,[None,10],name='label')\n",
    "cross_entropy=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y,logits=logits))\n",
    "l2_loss=tf.nn.l2_loss(w_1)+tf.nn.l2_loss(w_2)\n",
    "total_loss=cross_entropy+7e-5*l2_loss\n",
    "train_step=tf.train.GradientDescentOptimizer(learning_rate).minimize(total_loss)#GD优化器\n",
    "\n",
    "corret_prediction=tf.equal(tf.arg_max(y,1),tf.arg_max(logits,1))\n",
    "accuracy=tf.reduce_mean(tf.cast(corret_prediction,tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sess=tf.Session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------\n",
      "step:[100],entroty loss:[0.006230225786566734],l2_loss:[322.14447021484375],total_loss:[0.028780339285731316]\n",
      "1.0\n",
      "0.9803\n",
      "----------\n",
      "step:[200],entroty loss:[0.003974666818976402],l2_loss:[321.95611572265625],total_loss:[0.026511594653129578]\n",
      "1.0\n",
      "0.9804\n",
      "----------\n",
      "step:[300],entroty loss:[0.008368891663849354],l2_loss:[321.69891357421875],total_loss:[0.030887816101312637]\n",
      "1.0\n",
      "0.9802\n",
      "----------\n",
      "step:[400],entroty loss:[0.007941290736198425],l2_loss:[321.48095703125],total_loss:[0.030444959178566933]\n",
      "1.0\n",
      "0.9801\n",
      "----------\n",
      "step:[500],entroty loss:[0.005373595282435417],l2_loss:[321.32537841796875],total_loss:[0.02786637283861637]\n",
      "1.0\n",
      "0.9801\n",
      "----------\n",
      "step:[600],entroty loss:[0.004509435500949621],l2_loss:[321.1989440917969],total_loss:[0.026993362233042717]\n",
      "1.0\n",
      "0.9794\n",
      "----------\n",
      "step:[700],entroty loss:[0.004775148816406727],l2_loss:[320.99169921875],total_loss:[0.02724456787109375]\n",
      "1.0\n",
      "0.9801\n",
      "----------\n",
      "step:[800],entroty loss:[0.003198186168447137],l2_loss:[320.85540771484375],total_loss:[0.02565806545317173]\n",
      "1.0\n",
      "0.979\n",
      "----------\n",
      "step:[900],entroty loss:[0.006454796064645052],l2_loss:[320.66668701171875],total_loss:[0.028901463374495506]\n",
      "1.0\n",
      "0.9793\n",
      "----------\n",
      "step:[1000],entroty loss:[0.0032845011446624994],l2_loss:[320.4383239746094],total_loss:[0.025715183466672897]\n",
      "1.0\n",
      "0.9793\n",
      "----------\n",
      "step:[1100],entroty loss:[0.009103910066187382],l2_loss:[320.2460632324219],total_loss:[0.031521134078502655]\n",
      "1.0\n",
      "0.9794\n",
      "----------\n",
      "step:[1200],entroty loss:[0.008927293121814728],l2_loss:[320.063232421875],total_loss:[0.03133171796798706]\n",
      "1.0\n",
      "0.9798\n",
      "----------\n",
      "step:[1300],entroty loss:[0.004660882521420717],l2_loss:[319.967529296875],total_loss:[0.027058608829975128]\n",
      "1.0\n",
      "0.9797\n",
      "----------\n",
      "step:[1400],entroty loss:[0.0028013980481773615],l2_loss:[319.7185974121094],total_loss:[0.02518169954419136]\n",
      "1.0\n",
      "0.9797\n",
      "----------\n",
      "step:[1500],entroty loss:[0.004881676286458969],l2_loss:[319.46710205078125],total_loss:[0.02724437415599823]\n",
      "1.0\n",
      "0.9794\n",
      "----------\n",
      "step:[1600],entroty loss:[0.005498138722032309],l2_loss:[319.27606201171875],total_loss:[0.02784746326506138]\n",
      "1.0\n",
      "0.9789\n",
      "----------\n",
      "step:[1700],entroty loss:[0.0073923468589782715],l2_loss:[319.1200256347656],total_loss:[0.029730748385190964]\n",
      "1.0\n",
      "0.9801\n",
      "----------\n",
      "step:[1800],entroty loss:[0.0036657890304923058],l2_loss:[319.0147705078125],total_loss:[0.025996822863817215]\n",
      "1.0\n",
      "0.9798\n",
      "----------\n",
      "step:[1900],entroty loss:[0.005181602202355862],l2_loss:[318.88275146484375],total_loss:[0.027503393590450287]\n",
      "1.0\n",
      "0.98\n",
      "----------\n",
      "step:[2000],entroty loss:[0.0030182497575879097],l2_loss:[318.646484375],total_loss:[0.025323502719402313]\n",
      "1.0\n",
      "0.9805\n",
      "----------\n",
      "step:[2100],entroty loss:[0.0037677150685340166],l2_loss:[318.5588073730469],total_loss:[0.02606683224439621]\n",
      "1.0\n",
      "0.98\n",
      "----------\n",
      "step:[2200],entroty loss:[0.013970701955258846],l2_loss:[318.4842529296875],total_loss:[0.03626460209488869]\n",
      "1.0\n",
      "0.9796\n",
      "----------\n",
      "step:[2300],entroty loss:[0.007911440916359425],l2_loss:[318.41339111328125],total_loss:[0.030200377106666565]\n",
      "1.0\n",
      "0.9805\n",
      "----------\n",
      "step:[2400],entroty loss:[0.006849737837910652],l2_loss:[318.3165283203125],total_loss:[0.029131894931197166]\n",
      "1.0\n",
      "0.9803\n",
      "----------\n",
      "step:[2500],entroty loss:[0.0028064509388059378],l2_loss:[318.2225646972656],total_loss:[0.02508203126490116]\n",
      "1.0\n",
      "0.9799\n",
      "----------\n",
      "step:[2600],entroty loss:[0.005255454685539007],l2_loss:[318.1282958984375],total_loss:[0.02752443589270115]\n",
      "1.0\n",
      "0.9803\n",
      "----------\n",
      "step:[2700],entroty loss:[0.004605559632182121],l2_loss:[318.04254150390625],total_loss:[0.026868538931012154]\n",
      "1.0\n",
      "0.98\n",
      "----------\n",
      "step:[2800],entroty loss:[0.003235923359170556],l2_loss:[317.95361328125],total_loss:[0.025492677465081215]\n",
      "1.0\n",
      "0.9798\n",
      "----------\n",
      "step:[2900],entroty loss:[0.0032502084504812956],l2_loss:[317.8599853515625],total_loss:[0.025500407442450523]\n",
      "1.0\n",
      "0.9792\n",
      "----------\n",
      "step:[3000],entroty loss:[0.004491095431149006],l2_loss:[317.76104736328125],total_loss:[0.0267343707382679]\n",
      "1.0\n",
      "0.9798\n",
      "----------\n",
      "step:[3100],entroty loss:[0.004067614208906889],l2_loss:[317.660888671875],total_loss:[0.02630387805402279]\n",
      "1.0\n",
      "0.9798\n",
      "----------\n",
      "step:[3200],entroty loss:[0.004250319674611092],l2_loss:[317.5864562988281],total_loss:[0.026481371372938156]\n",
      "1.0\n",
      "0.9797\n",
      "----------\n",
      "step:[3300],entroty loss:[0.005181091837584972],l2_loss:[317.4952392578125],total_loss:[0.0274057574570179]\n",
      "1.0\n",
      "0.979\n",
      "----------\n",
      "step:[3400],entroty loss:[0.004317951388657093],l2_loss:[317.38946533203125],total_loss:[0.026535212993621826]\n",
      "1.0\n",
      "0.98\n",
      "----------\n",
      "step:[3500],entroty loss:[0.00512932101264596],l2_loss:[317.31146240234375],total_loss:[0.027341123670339584]\n",
      "1.0\n",
      "0.9803\n",
      "----------\n",
      "step:[3600],entroty loss:[0.005494580138474703],l2_loss:[317.2137756347656],total_loss:[0.0276995450258255]\n",
      "1.0\n",
      "0.9805\n",
      "----------\n",
      "step:[3700],entroty loss:[0.005479159764945507],l2_loss:[317.1307067871094],total_loss:[0.027678310871124268]\n",
      "1.0\n",
      "0.9798\n",
      "----------\n",
      "step:[3800],entroty loss:[0.005743319168686867],l2_loss:[317.0404052734375],total_loss:[0.027936147525906563]\n",
      "1.0\n",
      "0.9799\n",
      "----------\n",
      "step:[3900],entroty loss:[0.0044023129157722],l2_loss:[316.9507141113281],total_loss:[0.026588864624500275]\n",
      "1.0\n",
      "0.9793\n",
      "----------\n",
      "step:[4000],entroty loss:[0.005249993409961462],l2_loss:[316.86376953125],total_loss:[0.02743045799434185]\n",
      "1.0\n",
      "0.98\n",
      "----------\n",
      "step:[4100],entroty loss:[0.0068952711299061775],l2_loss:[316.8127746582031],total_loss:[0.029072165489196777]\n",
      "1.0\n",
      "0.9799\n",
      "----------\n",
      "step:[4200],entroty loss:[0.006820743903517723],l2_loss:[316.7579345703125],total_loss:[0.028993800282478333]\n",
      "1.0\n",
      "0.9795\n",
      "----------\n",
      "step:[4300],entroty loss:[0.0046465168707072735],l2_loss:[316.7118225097656],total_loss:[0.026816345751285553]\n",
      "1.0\n",
      "0.9797\n",
      "----------\n",
      "step:[4400],entroty loss:[0.0035315463319420815],l2_loss:[316.6690368652344],total_loss:[0.025698378682136536]\n",
      "1.0\n",
      "0.9797\n",
      "----------\n",
      "step:[4500],entroty loss:[0.004983648657798767],l2_loss:[316.61309814453125],total_loss:[0.027146566659212112]\n",
      "1.0\n",
      "0.9803\n",
      "----------\n",
      "step:[4600],entroty loss:[0.003887313650920987],l2_loss:[316.5726318359375],total_loss:[0.026047399267554283]\n",
      "1.0\n",
      "0.9801\n",
      "----------\n",
      "step:[4700],entroty loss:[0.0034519443288445473],l2_loss:[316.52606201171875],total_loss:[0.025608770549297333]\n",
      "1.0\n",
      "0.9799\n",
      "----------\n",
      "step:[4800],entroty loss:[0.004475592169910669],l2_loss:[316.47845458984375],total_loss:[0.02662908472120762]\n",
      "1.0\n",
      "0.9801\n",
      "----------\n",
      "step:[4900],entroty loss:[0.0027718031778931618],l2_loss:[316.43365478515625],total_loss:[0.02492215856909752]\n",
      "1.0\n",
      "0.9802\n",
      "----------\n",
      "step:[5000],entroty loss:[0.0035622301511466503],l2_loss:[316.3817138671875],total_loss:[0.02570895105600357]\n",
      "1.0\n",
      "0.9806\n",
      "----------\n",
      "step:[5100],entroty loss:[0.008786502294242382],l2_loss:[316.37139892578125],total_loss:[0.030932500958442688]\n",
      "1.0\n",
      "0.9801\n",
      "----------\n",
      "step:[5200],entroty loss:[0.003653069492429495],l2_loss:[316.36138916015625],total_loss:[0.025798367336392403]\n",
      "1.0\n",
      "0.9798\n",
      "----------\n",
      "step:[5300],entroty loss:[0.004699046723544598],l2_loss:[316.3513488769531],total_loss:[0.026843640953302383]\n",
      "1.0\n",
      "0.98\n",
      "----------\n",
      "step:[5400],entroty loss:[0.0038844975642859936],l2_loss:[316.3414306640625],total_loss:[0.026028398424386978]\n",
      "1.0\n",
      "0.9798\n",
      "----------\n",
      "step:[5500],entroty loss:[0.003089283127337694],l2_loss:[316.3310852050781],total_loss:[0.025232458487153053]\n",
      "1.0\n",
      "0.9797\n",
      "----------\n",
      "step:[5600],entroty loss:[0.0035613076761364937],l2_loss:[316.3214111328125],total_loss:[0.025703806430101395]\n",
      "1.0\n",
      "0.98\n",
      "----------\n",
      "step:[5700],entroty loss:[0.003297368995845318],l2_loss:[316.3121643066406],total_loss:[0.025439221411943436]\n",
      "1.0\n",
      "0.9799\n",
      "----------\n",
      "step:[5800],entroty loss:[0.003654859261587262],l2_loss:[316.3009338378906],total_loss:[0.02579592540860176]\n",
      "1.0\n",
      "0.9799\n",
      "----------\n",
      "step:[5900],entroty loss:[0.006128078326582909],l2_loss:[316.29302978515625],total_loss:[0.028268590569496155]\n",
      "1.0\n",
      "0.9799\n",
      "----------\n",
      "step:[6000],entroty loss:[0.003483370877802372],l2_loss:[316.28436279296875],total_loss:[0.025623276829719543]\n",
      "1.0\n",
      "0.9802\n"
     ]
    }
   ],
   "source": [
    "lr=1.0\n",
    "for step in range(6000):\n",
    "    if step<2000:\n",
    "        lr=0.3\n",
    "    elif step<4000:\n",
    "        lr=0.1\n",
    "    elif step<5000:\n",
    "        lr=0.05\n",
    "    else:\n",
    "        lr=0.01\n",
    "    batch_x,batch_y=mnist.train.next_batch(100)\n",
    "    _,loss,l2_loss_value,total_loss_value=sess.run([train_step,cross_entropy,l2_loss,total_loss],feed_dict={x:batch_x,y:batch_y,learning_rate:lr})\n",
    "    if (step+1)%100==0:\n",
    "        print('-'*10)\n",
    "        print('step:[{}],entroty loss:[{}],l2_loss:[{}],total_loss:[{}]'.format(step+1,loss,l2_loss_value,total_loss_value))\n",
    "        print(sess.run(accuracy,feed_dict={x:batch_x,y:batch_y}))\n",
    "        print(sess.run(accuracy,feed_dict={x:mnist.test.images,y:mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "修改隐层数量：增加w，b个数，在size正确的情况下增加多个logits=w*x+b，同时对隐层使用激活函数传递至下一层。增加隐层的数量能使重要特征被提取出来。方便分类器判定。\n",
    "修改神经元个数：对L1_units_count初始化是可以设定神经元个数，在w和b的设定中是的神经元个数满足矩阵关系即可，增加神经元个数能提取更多的特征向量出来，传入下一个隐层提取处关键的特征。\n",
    "在计算损失函数的时候。添加正则项，标注出来每一层的正则化类型，与原来的交叉熵损失相加得到总损失。正则项可以有效的防止训练数据过拟合，加快训练过程。\n",
    "不同的初始化可以使得模型更好的适配训练数据，更快的找到最优解。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
